bitkeeper revision 1.865 (407e933bgy9845pPJrw2YFAvCNIgEA)

More TLB-flush cleanups. Simplify and rationalise the interface.

diff --git a/xen/arch/i386/flushtlb.c b/xen/arch/i386/flushtlb.c
index 9180454e4e8613fac18129e866305fc682d2c532..8712682d36358c9cdc9d2069f217c6c899a07e23 100644 (file)
--- a/xen/arch/i386/flushtlb.c
+++ b/xen/arch/i386/flushtlb.c
@@ -14,7 +14,7 @@
 u32 tlbflush_clock;
 u32 tlbflush_time[NR_CPUS];
 
-static inline void tlb_clocktick(unsigned int cpu)
+void tlb_clocktick(void)
 {
     u32 y, ny;
 
@@ -34,23 +34,6 @@ static inline void tlb_clocktick(unsigned int cpu)
     }
     while ( unlikely((ny = cmpxchg(&tlbflush_clock, y-1, y)) != y-1) );
 
-    /* Update cpu's timestamp to new time. */
-    tlbflush_time[cpu] = y;
+    /* Update this CPU's timestamp to new time. */
+    tlbflush_time[smp_processor_id()] = y;
 }
-
-void write_cr3_counted(unsigned long pa)
-{
-    __asm__ __volatile__ ( 
-        "movl %0, %%cr3"
-        : : "r" (pa) : "memory" );
-    tlb_clocktick(smp_processor_id());
-}
-
-void flush_tlb_counted(void)
-{
-    __asm__ __volatile__ ( 
-        "movl %%cr3, %%eax; movl %%eax, %%cr3"
-        : : : "memory", "eax" );
-    tlb_clocktick(smp_processor_id());
-}
-

diff --git a/xen/arch/i386/process.c b/xen/arch/i386/process.c
index a9c2efd06dff75033da4e7b16b22d242ae7d1a01..ea5c51d17636615d4799ab2bdbafb84f3c8aefce 100644 (file)
--- a/xen/arch/i386/process.c
+++ b/xen/arch/i386/process.c
@@ -280,8 +280,9 @@ void switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         }
     }
 
-    /* Switch page tables.  */
-    write_ptbase( &next_p->mm );
+    /* Switch page tables. */
+    write_ptbase(&next_p->mm);
+    tlb_clocktick();
 
     set_current(next_p);
 

diff --git a/xen/arch/i386/setup.c b/xen/arch/i386/setup.c
index 0eb4466291a1087f91cc62f86bcb8b3407ec8332..c9eccfeb1d50346b40d3aad6790f5bd97a0536c7 100644 (file)
--- a/xen/arch/i386/setup.c
+++ b/xen/arch/i386/setup.c
@@ -282,8 +282,7 @@ void __init cpu_init(void)
 #undef CD
 
     /* Install correct page table. */
-    __asm__ __volatile__ ("movl %%eax,%%cr3"
-                          : : "a" (pagetable_val(current->mm.pagetable)));
+    write_ptbase(&current->mm);
 
     init_idle_task();
 }

diff --git a/xen/common/domain.c b/xen/common/domain.c
index 7eaa5628ad5615e0b9b5a1feffc0c8decf972091..053dfecef45a392c983550ca142f4acfb481d86b 100644 (file)
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -929,7 +929,7 @@ int construct_dom0(struct task_struct *p,
 
     /* Install the new page tables. */
     __cli();
-    write_cr3_counted(pagetable_val(p->mm.pagetable));
+    write_ptbase(&p->mm);
 
     /* Copy the OS image. */
     (void)loadelfimage(image_start);
@@ -977,7 +977,7 @@ int construct_dom0(struct task_struct *p,
     *dst = '\0';
 
     /* Reinstate the caller's page tables. */
-    write_cr3_counted(pagetable_val(current->mm.pagetable));
+    write_ptbase(&current->mm);
     __sti();
 
     /* Destroy low mappings - they were only for our convenience. */

diff --git a/xen/common/memory.c b/xen/common/memory.c
index 4b13f84fb33fd9493bedc1608acf18df27e6f496..7c94748e073093f78fa3a536d6382550bb9de0af 100644 (file)
--- a/xen/common/memory.c
+++ b/xen/common/memory.c
@@ -871,6 +871,13 @@ static int do_extended_command(unsigned long ptr, unsigned long val)
             write_ptbase(&current->mm);
 
             put_page_and_type(&frame_table[old_base_pfn]);    
+
+            /*
+             * Note that we tick the clock /after/ dropping the old base's
+             * reference count. If the page tables got freed then this will
+             * avoid unnecessary TLB flushes when the pages are reused.
+             */
+            tlb_clocktick();
         }
         else
         {
@@ -1096,9 +1103,7 @@ int do_mmu_update(mmu_update_t *ureqs, int count)
     percpu_info[cpu].deferred_ops = 0;
 
     if ( deferred_ops & DOP_FLUSH_TLB )
-    {
-        write_ptbase(&current->mm);
-    }
+        local_flush_tlb();
 
     if ( deferred_ops & DOP_RELOAD_LDT )
         (void)map_ldt_shadow_page(0);
@@ -1168,9 +1173,7 @@ int do_update_va_mapping(unsigned long page_nr,
 
     if ( unlikely(deferred_ops & DOP_FLUSH_TLB) || 
          unlikely(flags & UVMF_FLUSH_TLB) )
-    {
-        write_ptbase(&p->mm);
-    }
+        local_flush_tlb();
     else if ( unlikely(flags & UVMF_INVLPG) )
         __flush_tlb_one(page_nr << PAGE_SHIFT);
 

diff --git a/xen/include/asm-i386/flushtlb.h b/xen/include/asm-i386/flushtlb.h
index 4b558eae834ffb709a0af86d403c94f8d01b23aa..f0d4bb946c3b1580a700c6cbb72f628311b633e2 100644 (file)
--- a/xen/include/asm-i386/flushtlb.h
+++ b/xen/include/asm-i386/flushtlb.h
@@ -43,9 +43,7 @@ static inline int NEED_FLUSH(u32 cpu_stamp, u32 lastuse_stamp)
 extern u32 tlbflush_clock;
 extern u32 tlbflush_time[NR_CPUS];
 
+extern void tlb_clocktick(void);
 extern void new_tlbflush_clock_period(void);
 
-extern void write_cr3_counted(unsigned long pa);
-extern void flush_tlb_counted(void);
-
 #endif /* __FLUSHTLB_H__ */

diff --git a/xen/include/asm-i386/page.h b/xen/include/asm-i386/page.h
index 54cbd85f1147b1831a1e462220c72f522fed67f4..61996d4ccccf0bed1a32f392a12871680d383dfe 100644 (file)
--- a/xen/include/asm-i386/page.h
+++ b/xen/include/asm-i386/page.h
@@ -98,7 +98,13 @@ typedef struct { unsigned long pt_lo; } pagetable_t;
 extern l2_pgentry_t idle_pg_table[ENTRIES_PER_L2_PAGETABLE];
 extern void paging_init(void);
 
-#define __flush_tlb() flush_tlb_counted()
+#define __flush_tlb()                                    \
+    do {                                                 \
+        __asm__ __volatile__ (                           \
+            "movl %%cr3, %%eax; movl %%eax, %%cr3"       \
+            : : : "memory", "eax" );                     \
+        tlb_clocktick();                                 \
+    } while ( 0 )
 
 /* Flush global pages as well. */
 
@@ -120,7 +126,7 @@ extern void paging_init(void);
 #define __flush_tlb_pge()                                              \
        do {                                                            \
                 __pge_off();                                            \
-               flush_tlb_counted();                                    \
+               __flush_tlb();                                          \
                 __pge_on();                                             \
        } while (0)
 

diff --git a/xen/include/asm-i386/processor.h b/xen/include/asm-i386/processor.h
index cbce688e8b339dc45522b1001c031b3464df097e..26f64d1f9f36c264b2998b0c3156722e6d28affa 100644 (file)
--- a/xen/include/asm-i386/processor.h
+++ b/xen/include/asm-i386/processor.h
@@ -209,9 +209,6 @@ static inline unsigned int cpuid_edx(unsigned int op)
 #define X86_CR4_OSFXSR         0x0200  /* enable fast FPU save and restore */
 #define X86_CR4_OSXMMEXCPT     0x0400  /* enable unmasked SSE exceptions */
 
-#define load_cr3(pgdir) \
-       asm volatile("movl %0,%%cr3": :"r" (__pa(pgdir)));
-
 /*
  * Save the cr4 feature set we're using (ie
  * Pentium 4MB enable and PPro Global page
@@ -461,12 +458,16 @@ struct mm_struct {
     char gdt[6];
 };
 
-static inline void write_ptbase( struct mm_struct *m )
+static inline void write_ptbase(struct mm_struct *mm)
 {
-    if ( unlikely(m->shadow_mode) )
-        write_cr3_counted(pagetable_val(m->shadow_table));
+    unsigned long pa;
+
+    if ( unlikely(mm->shadow_mode) )
+        pa = pagetable_val(mm->shadow_table);
     else
-        write_cr3_counted(pagetable_val(m->pagetable));
+        pa = pagetable_val(mm->pagetable);
+
+    __asm__ __volatile__ ( "movl %0, %%cr3" : : "r" (pa) : "memory" );
 }
 
 #define IDLE0_MM                                                    \

diff --git a/xen/include/asm-x86_64/flushtlb.h b/xen/include/asm-x86_64/flushtlb.h
index 4b558eae834ffb709a0af86d403c94f8d01b23aa..f0d4bb946c3b1580a700c6cbb72f628311b633e2 100644 (file)
--- a/xen/include/asm-x86_64/flushtlb.h
+++ b/xen/include/asm-x86_64/flushtlb.h
@@ -43,9 +43,7 @@ static inline int NEED_FLUSH(u32 cpu_stamp, u32 lastuse_stamp)
 extern u32 tlbflush_clock;
 extern u32 tlbflush_time[NR_CPUS];
 
+extern void tlb_clocktick(void);
 extern void new_tlbflush_clock_period(void);
 
-extern void write_cr3_counted(unsigned long pa);
-extern void flush_tlb_counted(void);
-
 #endif /* __FLUSHTLB_H__ */

diff --git a/xen/include/asm-x86_64/page.h b/xen/include/asm-x86_64/page.h
index 0073a45c938c0056e390efdb4ddbf873f97be993..cb8651ec8a2a7bf334acf6a0d1430e0e4b3bfba7 100644 (file)
--- a/xen/include/asm-x86_64/page.h
+++ b/xen/include/asm-x86_64/page.h
@@ -162,7 +162,13 @@ extern unsigned long vm_force_exec32;
 extern l2_pgentry_t idle_pg_table[ENTRIES_PER_L2_PAGETABLE];
 extern void paging_init(void);
 
-#define __flush_tlb() flush_tlb_counted()
+#define __flush_tlb()                                    \
+    do {                                                 \
+        __asm__ __volatile__ (                           \
+            "movl %%cr3, %%eax; movl %%eax, %%cr3"       \
+            : : : "memory", "eax" );                     \
+        tlb_clocktick();                                 \
+    } while ( 0 )
 
 /* Flush global pages as well. */
 
@@ -184,7 +190,7 @@ extern void paging_init(void);
 #define __flush_tlb_pge()                                              \
        do {                                                            \
                 __pge_off();                                            \
-               flush_tlb_counted();                                    \
+               __flush_tlb();                                          \
                 __pge_on();                                             \
        } while (0)